from django.http import HttpResponse
from django.conf import settings
import json
import logging
import sqlite3
import os
import datetime
from xml.etree import ElementTree as ET
from xml.dom.minidom import Document
import time
from concurrent.futures import ThreadPoolExecutor,as_completed,ProcessPoolExecutor

logging.basicConfig(level=logging.DEBUG,format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',datefmt='%a, %d %b %Y %H:%M:%S')
execShellDir='/export/datamarket/OozieProperties/oozie_project/oozie_shell/upload'
oozie_home='/export/datamarket/oozie/oozie-4.3.1/bin/'
hadoop_home='/export/datamarket/hadoop/bin/'
def basic(request):
	if request.method == 'POST':
		try:	
			received_json_data=json.loads(request.body.decode().replace("'", "\""))
			logging.info("\n\n\n\t"+str(received_json_data)+"\n\n")
			oozieMission=received_json_data['oozieMission'].strip()
			mission_val=received_json_data['oozieMission'].strip() 
			oozieTime=received_json_data['oozieTime'].strip() 
			time_val=received_json_data['oozieTime'].strip() 
			erpName=received_json_data['erpName'].strip() 
			erpName_val=received_json_data['erpName'].strip() 
			erpPassword=received_json_data['erpPassword'].strip() 
			erpPassword_val=received_json_data['erpPassword'].strip() 
			result={'result':oozieMission+'任务创建成功','code':0}
			result2={'result':oozieMission+'任务已经存在','code':1}
			
			
			today='2018-11-10'
			if os.path.exists("/export/datamarket/OozieExec/"+today+"/"+mission_val):
				logging.info("\n\n\n\t"+mission_val+"任务已存在\n\n")
				logging.info(result2)
				
				return HttpResponse(json.dumps(result2,ensure_ascii=False), content_type="application/json;charset=utf-8")
			else:
				logging.info("\n\n\n\t"+str(result)+"\n\n")
				logging.info(result)
				#save_oozie_info(mission_val, time_val,erpName_val,erpPassword_val)
				#脚本对应ID
				shellID=received_json_data['shellID']
				save_oozie_info(mission_val, time_val,erpName_val,erpPassword_val)
				db_path = os.path.normpath('/export/datamarket/OozieProperties/oozie_project/db.sqlite3')  
				db = sqlite3.connect(db_path) 
				# 通过cursor创建游标
				cursor = db.cursor()
				sql_cmd=  "select * from oozie_shell where shellID='{}'".format(shellID) 
				cursor.execute(sql_cmd)
				shellID=cursor.fetchone()
				oozieShellName=shellID[1]
				sql_update_shellName="update oozie_configuration set shellName='{}' where mission_val='{}'".format(oozieShellName,mission_val)
				cursor.execute(sql_update_shellName)
				logging.info("更新sqlite中任务"+mission_val+"对应脚本"+oozieShellName)
				sql_update_shellID="update oozie_configuration set shellID='{}' where mission_val='{}'".format(shellID[0],mission_val)
				logging.info(sql_update_shellID)
				cursor.execute(sql_update_shellID)
				logging.info("更新sqlite中任务"+mission_val+"对应脚本ID"+str(shellID[0]))

				cursor.close()
				db.commit()
				db.close()
				#oozie提交任务
				oozieShell=execShellDir+"/"+oozieShellName
				run(mission_val,oozieShell)
				return HttpResponse(json.dumps(result,ensure_ascii=False), content_type="application/json;charset=utf-8")

		except:
			logging.info('\n\n\n\t输入格式有误\n\n')
			resultError={'result':'上传失败,请检查格式','code':2}
			return HttpResponse(json.dumps(resultError,ensure_ascii=False), content_type="application/json;charset=utf-8")
	else:
		pass


def save_oozie_info(mission_val, time_val,erpName_val,erpPassword_val):
        mission_val=mission_val
        time_val=time_val
        erpName_val=erpName_val
        erpPassword_val=erpPassword_val
        db_path = os.path.normpath('/export/datamarket/OozieProperties/oozie_project/db.sqlite3')  
        db = sqlite3.connect(db_path) 
        
        # 通过cursor创建游标
        cursor = db.cursor()
        sql_cmd = '''select * from oozie_configuration where mission_val = '%s' ''' % mission_val
        cursor.execute(sql_cmd)
        res = cursor.fetchall()
        suc = True
        if len(res) > 0 :
                logging.info('%s 任务已存在，无需插入' % mission_val)
                suc = True
        else:
                sql_cmd_2 =  "insert into oozie_configuration(jobid,mission_val, time_val,erpName_val,erpPassword_val) values(null,'{}', '{}','{}','{}')".format(mission_val, time_val,erpName_val,erpPassword_val) 
                cursor.execute(sql_cmd_2)
                logging.info('%s added 成功！' % mission_val)
                suc = False
        cursor.close()
        db.commit()
        db.close()
        return suc 

def main(todayfile,today,mission_val,time_val,erpName_val,erpPassword_val):
	today=today
	todayfile=todayfile
	mission_val=mission_val
	time_val=time_val
	erpName_val=erpName_val
	erpPassword_val=erpPassword_val
	fileList=[]
	jobproperties='job.properties'
	##oozie其他固定配置文件所在目录
	ooziePorperties='/export/datamarket/OoziePorperties/'
	#多action需要的
	oozieWorkflowAction='/export/datamarket/OozieScript/action'
	#生成workflow.xml文件
	def workflowCreate(mission_val,name,end,allFile):
		
		doc = Document()
		workflow = doc.createElement('workflow-app')
		workflow.setAttribute('xmlns','uri:oozie:workflow:0.4')
		workflow.setAttribute('name','%s'%mission_val)
		doc.appendChild(workflow)
		
		start = doc.createElement("start")
		start.setAttribute('to','%s'%name)
		workflow.appendChild(start)
		
		action=doc.createElement("action")
		action.setAttribute('name','%s'%name)
		action.setAttribute('retry-max','3')
		action.setAttribute('retry-interval','1')
		workflow.appendChild(action)
		
		shell=doc.createElement("shell")
		shell.setAttribute('xmlns','uri:oozie:shell-action:0.2')
		action.appendChild(shell)
		
		jobtracker=doc.createElement("job-tracker")
		shell.appendChild(jobtracker)
		jobtracker_name=doc.createTextNode("${jobTracker}")
		jobtracker.appendChild(jobtracker_name)
		
		namenode=doc.createElement("name-node")
		shell.appendChild(namenode)
		namenode_name=doc.createTextNode("${nameNode}")
		namenode.appendChild(namenode_name)
		
		prepare=doc.createElement("prepare")
		shell.appendChild(prepare)
		
		deletepath=doc.createElement("delete")
		deletepath.setAttribute('path','${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}/outputdir/')
		prepare.appendChild(deletepath)
		mkdirpath=doc.createElement("mkdir")
		mkdirpath.setAttribute('path','${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}/outputdir/')
		prepare.appendChild(mkdirpath)
		
		configuration=doc.createElement("configuration")
		shell.appendChild(configuration)
		property=doc.createElement("property")
		configuration.appendChild(property)
		property_tag_name=doc.createElement("name")
		property.appendChild(property_tag_name)
		property_tag_name_name=doc.createTextNode("mapred.job.queue.name")
		property_tag_name.appendChild(property_tag_name_name)
		property_tag_value=doc.createElement("value")
		property.appendChild(property_tag_value)
		property_tag_value_name=doc.createTextNode("${queueName}")
		property_tag_value.appendChild(property_tag_value_name)
		
		property2=doc.createElement("property")
		configuration.appendChild(property2)
		property2_tag_name=doc.createElement("name")
		property2.appendChild(property2_tag_name)
		property2_tag_name_name=doc.createTextNode("oozie.launcher.mapreduce.map.memory.mb")
		property2_tag_name.appendChild(property2_tag_name_name)
		property2_tag_value=doc.createElement("value")
		property2.appendChild(property2_tag_value)
		property2_tag_value_name=doc.createTextNode("8192")
		property2_tag_value.appendChild(property2_tag_value_name)
		
		property3=doc.createElement("property")
		configuration.appendChild(property3)
		property3_tag_name=doc.createElement("name")
		property3.appendChild(property3_tag_name)
		property3_tag_name_name=doc.createTextNode("oozie.launcher.mapreduce.reduce.memory.mb")
		property3_tag_name.appendChild(property3_tag_name_name)
		property3_tag_value=doc.createElement("value")
		property3.appendChild(property3_tag_value)
		property3_tag_value_name=doc.createTextNode("8192")
		property3_tag_value.appendChild(property3_tag_value_name)
		
		execExec=doc.createElement("exec")
		shell.appendChild(execExec)
		execExec_name=doc.createTextNode("%s.sh"%name)
		execExec.appendChild(execExec_name)
		
		file=doc.createElement("file")
		shell.appendChild(file)
		file_name=doc.createTextNode("${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}/%s.sh"%name)
		file.appendChild(file_name)
		
		ok=doc.createElement("ok")
		ok.setAttribute('to','%s'%end)
		action.appendChild(ok)
		error=doc.createElement("error")
		error.setAttribute('to','fail')
		action.appendChild(error)
		
		
		
		
		kill=doc.createElement("kill")
		kill.setAttribute("name","fail")
		workflow.appendChild(kill)
		
		message=doc.createElement("message")
		kill.appendChild(message)
		message_name=doc.createTextNode("Shell failed, error message[${wf:errorMessage(wf:lastErrorNode())}]")
		message.appendChild(message_name)
		
		end=doc.createElement("end")
		end.setAttribute("name","end")
		workflow.appendChild(end)
		
		with open(allFile+'/workflow.xml','w') as f:
			f.write(doc.toprettyxml(indent="  "))
			
		
	#生成coordinator.xml文件
	def coordinator(mission_val,allFile):
		doc = Document()
		coordinatorapp=doc.createElement("coordinator-app")
		coordinatorapp.setAttribute("xmlns","uri:oozie:coordinator:0.4")
		coordinatorapp.setAttribute("timezone","Asia/Shanghai")
		coordinatorapp.setAttribute("start","${start}")
		coordinatorapp.setAttribute("name","{}".format(mission_val))
		coordinatorapp.setAttribute("frequency","${frequency}")
		coordinatorapp.setAttribute("end","${end}")
		doc.appendChild(coordinatorapp)
		
		action=doc.createElement("action")
		coordinatorapp.appendChild(action)
		
		workflow=doc.createElement("workflow")
		action.appendChild(workflow)
		
		apppath=doc.createElement("app-path")
		workflow.appendChild(apppath)
		apppath_name=doc.createTextNode("${workflowAppUri}")
		apppath.appendChild(apppath_name)
		
		configuration=doc.createElement("configuration")
		workflow.appendChild(configuration)
		
		property=doc.createElement("property")
		configuration.appendChild(property)
		property_tag_name=doc.createElement("name")
		property.appendChild(property_tag_name)
		property_tag_name_name=doc.createTextNode("jobTracker")
		property_tag_name.appendChild(property_tag_name_name)
		property_tag_value=doc.createElement("value")
		property.appendChild(property_tag_value)
		property_tag_value_name=doc.createTextNode("${jobTracker}")
		property_tag_value.appendChild(property_tag_value_name)
		
		property2=doc.createElement("property")
		configuration.appendChild(property2)
		property2_tag_name=doc.createElement("name")
		property2.appendChild(property2_tag_name)
		property2_tag_name_name=doc.createTextNode("nameNode")
		property2_tag_name.appendChild(property2_tag_name_name)
		property2_tag_value=doc.createElement("value")
		property2.appendChild(property2_tag_value)
		property2_tag_value_name=doc.createTextNode("${nameNode}")
		property2_tag_value.appendChild(property2_tag_value_name)
		
		property3=doc.createElement("property")
		configuration.appendChild(property3)
		property3_tag_name=doc.createElement("name")
		property3.appendChild(property3_tag_name)
		property3_tag_name_name=doc.createTextNode("queueName")
		property3_tag_name.appendChild(property3_tag_name_name)
		property3_tag_value=doc.createElement("value")
		property3.appendChild(property3_tag_value)
		property3_tag_value_name=doc.createTextNode("${queueName}")
		property3_tag_value.appendChild(property3_tag_value_name)

		with open(allFile+'/coordinator.xml','w') as f:
			f.write(doc.toprettyxml(indent="  "))
		

	#删除文件
	def deleteFile(deleteFile):
		deleteFile=deleteFile
		if os.path.exists(deleteFile):
			os.remove(deleteFile)
		else:
			pass
	
	allFile=todayfile
	if (os.path.isdir(allFile)):
		
		#dirList=execPath.split('/')[-1]

		#allfileList.append(allFile)
		#allDir=allFile
		#os.system("rm "+allFile+"/ceshiduo.properties")
		execNum=1
		for file in os.listdir(allFile):
			if file.endswith('.sh'):
				fileList.append(file)
				#os.system("mkdir "+execPath+file.split(".")[0])
				#print("mkdir "+execPath+file.split(".")[0])
				#os.system("rm "+allFile+"/"+jobproperties)
				#定时执行的job.properties
				deletefile=allFile+"/"+jobproperties
				deleteFile(deletefile)
				#立即执行的job.properties
				deletefile=allFile+"/jobImmediately.properties"
				deleteFile(deletefile)
				execDir=allFile.split("/")[-1]
				execDir='execDir='+execDir
				with open(allFile+'/'+jobproperties,'a') as f:
					f.write(execDir+"\n")
				with open(allFile+'/jobImmediately.properties','a') as f:
					f.write(execDir+"\n")
	
				execNum+=1
			#elif file.startswith('time'):
			#	print("配置文件是："+file)
			else:
				pass
		
		execNum2=1
		nameList=[]
		#os.system("rm "+allFile+"/workflow.xml")
		workflowxml=allFile+"/workflow.xml"
		if os.path.exists(workflowxml):
			os.remove(workflowxml)
		else:
			pass
	
		#os.system("rm "+allFile+"/coordinator.xml")
		coordinatorxml=allFile+"/coordinator.xml"
		deleteFile(coordinatorxml)
		for file in os.listdir(allFile):
			if file.endswith('.sh'):
				fileList.append(file)
				execDir0=allFile.split("/")[-1]
				execDir='execDir='+execDir0
				name=file.split(".")[0]
				#print('name'+str(execNum2)+'='+name)
				actionname='EXEC'+str(execNum2)
				actionExecFile=actionname+'.sh'
				strName='EXEC'+str(execNum2)+'='+name+'.sh'
				
				nameList.append(name)
				nameList.sort()
				if actionname=='EXEC1':
					end='end'
					
					workflowCreate(mission_val,name,end,allFile)
					###检测workflow的有效性
					os.system(oozie_home+"oozie validate "+allFile+"/workflow.xml")
					#coordinator(allFile)
					coordinator(mission_val,allFile)
					
					####################################################时间
					#os.system("cp "+ooziePorperties+"/time.properties "+allFile+"/")
					#os.system("cat "+allFile+"/time.properties >> "+allFile+"/"+jobproperties)
					#print(jobproperties)
					#print("os.system(\"cat \"+allFile+\"/time.properties >> \"+jobproperties)")						

				else:
					for end in nameList[1:]:
						nameList2=['end']
						endList =nameList[1:]+nameList2
						
							
					name=nameList[0]
					name="'"+name+"'"
					#print(name)
					os.chdir(allFile+"/")
					#os.system("if [ -f "action1 action2 action3 action4 action5 workflowCreateAction.py" ];then rm action1 action2 action3 action4 action5 workflowCreateAction.py fi")
					action1='action1'
					deleteFile(action1)
					action2='action2'
					deleteFile(action2)
					action3='action3'
					deleteFile(action3)
					action4='action4'
					deleteFile(action4)
					action5='action5'
					deleteFile(action5)
					deletefile='workflowCreateAction.py'
					deleteFile(deletefile)
					os.system("cp "+oozieWorkflowAction+"/action1 .")	
					os.system("cp "+oozieWorkflowAction+"/action3 .")	
					os.system("cp "+oozieWorkflowAction+"/action5 .")	
					with open(allFile+'/action2','w') as f:
						f.write('name='+name+"\n")
					#workflow(name,end)
					nameend=dict(zip(nameList,endList))
					#print(nameend)
					#keys.sort() 
					with open(allFile+'/actionname.txt','w') as f:
						for key in sorted(nameend.keys()):
							#sorted(nameend.items(), key=lambda d:d[1],reverse=True)
							f.write('workflowCreate(\'%s\',\'%s\')\n'%(key,nameend[key]))
					lineList=[]
					with open(allFile+'/actionname.txt','r') as r:
						for line in r.readlines():
							#print(line)
							#line=line.split('\n')
							lineList.append(line)
						lineSort=lineList[:]
						lineSort.sort()
						#print(lineSort)
						lineSortStr='\n'.join(lineSort)
						#print(lineSortStr)
					#workflowCreateStr='workflowCreateStr'.replace('workflowCreateStr',lineSortStr)
					with open(allFile+'/action4','w') as f:
						f.write(lineSortStr)
					os.system("cat "+allFile+"/action1 action2 action3 action4 action5>> workflowCreateAction.py")
					os.system("chmod +x "+allFile+"/workflowCreateAction.py")
					os.system("/usr/local/bin/python3 "+allFile+"/workflowCreateAction.py")
	
					####检测workflow的有效性
					os.system(oozie_home+"oozie validate "+allFile+"/workflow.xml")
					#coordinator(allFile)
					coordinator(mission_val,allFile)
				execNum2+=1
				
			else:
				pass
				#print('NO execfile !')
		print("共执行："+str(execNum2-1)+"个.sh文件")
		
		with open(allFile+'/'+jobproperties,'a') as f:
			###定时执行
			f.write("today="+today+"\nnameNode=hdfs://host-11-3-68-6:9000\njobTracker=host-11-3-68-6:8130\nqueueName=default\noozie.coord.application.path=${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}\nworkflowAppUri=${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}\noutputDir=${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}/outputdir/\n")
		with open(allFile+'/jobImmediately.properties','a') as f:
			###只执行workflow一次
			f.write("today="+today+"\nnameNode=hdfs://host-11-3-68-6:9000\njobTracker=host-11-3-68-6:8130\nqueueName=default\noozie.wf.application.path=${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}\noutputDir=${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}/outputdir/\n")
		####################################################时间
		#os.system("cp "+ooziePorperties+"/time.properties "+allFile+"/")
		#os.system("cat "+allFile+"/time.properties >> "+allFile+"/"+jobproperties)
		####################################################时间
		#specificTimeDate=datetime.datetime.now().strftime('%Y-%m-%d')
		os.chdir(allFile+"/")
		#os.system("rm action1 action2 action3 action4 action5 actionname.txt workflowCreateAction.py")
		action1='action1'
		deleteFile(action1)
		action2='action2'
		deleteFile(action2)
		action3='action3'
		deleteFile(action3)
		action4='action4'
		deleteFile(action4)
		action5='action5'
		deleteFile(action5)
		actionname='actionname.txt'
		deleteFile(actionname)
		deletefile='workflowCreateAction.py'
		deleteFile(deletefile)
		specificTime=(datetime.datetime.now() + datetime.timedelta(hours=0)).strftime('%Y-%m-%d %H:%M')
		
		
		########################################
		logging.info("oozie任务名称："+mission_val)
		logging.info("脚本执行时间："+time_val)
		logging.info("erp名字："+erpName_val)
		logging.info("erp密码："+erpPassword_val)
		with open(allFile+"/time.properties","w") as f:
			f.write("start="+specificTime[0:10]+"T"+specificTime[11:16]+"+0800\n")
			#f.write("start="+startTime)
			f.write("end=2222-02-22T22:22+0800\n")
			f.write("frequency={}".format(time_val))#前端传入
		#os.system("cp "+ooziePorperties+"/time.properties "+allFile+"/")
		os.system("cat "+allFile+"/time.properties >> "+allFile+"/"+jobproperties)
	
	
		####上传配置文件：
		os.system(hadoop_home+"hadoop fs -mkdir -p /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/")
		os.system(hadoop_home+"hadoop fs -rm -r /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/*")
		os.system(hadoop_home+"hadoop fs -put -f "+allFile+"/workflow.xml coordinator.xml *.sh /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/")
		#os.system(hadoop_home+"hadoop fs -put -f  /export/datamarket/OozieScript/lib /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/")
		#os.system(hadoop_home+"hadoop fs -put "+ooziePorperties+"hive-site.xml /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/")
		os.system(hadoop_home+"hadoop fs -ls /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/")
		os.system(oozie_home+"oozie job -config "+allFile+"/job.properties -run")
	else:
		pass
	



def run(mission_val,oozieShell):
	#mission_val='mission_val'
	#获取oozie任务的相关信息
	db_path = os.path.normpath('/export/datamarket/OozieProperties/oozie_project/db.sqlite3') 
	try: 
		conn = sqlite3.connect(db_path) 
		cursor = conn.cursor()
		sql = "select * from oozie_configuration where mission_val='{}'".format(mission_val) 
		try:
			cursor.execute(sql)
			allresult=cursor.fetchone()
			mission_val=allresult[1]
			time_val=allresult[2]
			erpName_val=allresult[3]
			erpPassword_val=allresult[4]
			# 提交SQL
			conn.commit()
			
		except:
			#发生错误时回滚
			conn.rollback()
			# 关闭数据库连接
		conn.close()
	except Exception as e: 
		print ('------', str(e)) 
		try: 
			conn.close() 
		except Exception as e: 
			pass
	today=datetime.datetime.now().strftime('%Y-%m-%d')
	today='2018-11-10'
	#print("\033[1;34;42m今天日期为："+today+"\033[0m")
	fileList=[]

	##需要执行的文件的目录所在
	os.system("mkdir -p /export/datamarket/OozieExec/"+today+"/"+mission_val)
	os.system("rm /export/datamarket/OozieExec/"+today+"/"+mission_val+"/*.sh")
	os.system("cp "+oozieShell+" /export/datamarket/OozieExec/"+today+"/"+mission_val) 
	os.system("chmod +x /export/datamarket/OozieExec/"+today+"/"+mission_val+"/*.sh") 
	execPath='/export/datamarket/OozieExec/'+today+'/'+mission_val
	
	allfileList=[]
	
	if not os.listdir(execPath):
		print("目录为空")
	else:
		#dirList=os.listdir(execPath)
	
		start = time.time()
		allfileList=[execPath]
		#main(todayfile,today,execPath,mission_val,time_val,erpName_val,erpPassword_val)
		#之前是为了同时提交并自动配置多个任务，所以用进程池
		with ProcessPoolExecutor(max_workers=32) as pool:
			futures =[pool.submit(main, todayfile,today,mission_val,time_val,erpName_val,erpPassword_val) for todayfile in allfileList]
			for future in futures:
				logging.info('\n\n\n\t执行中:%s, 已完成:%s\n\n' % (future.running(), future.done()))
	
			for future in as_completed(futures):
				print ('############################################# 分界线 ######################################################')
				logging.info('\n\n\n\t执行中:%s, 已完成:%s\n\n' % (future.running(), future.done()))
		end = time.time()
	
		logging.info('\n\n\n\t\033[1;34;42m提交任务用时 %.3f seconds.\033[0m\n\n' % (end - start))
		allDir='\n'.join(allfileList)
		logging.info("\n\n\n\033[1;34;42m当前执行目录为：\n"+allDir+"\033[0m\n\n")

